{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Optimize training ImageNet by distributing jobs across 4 GPUs as evenly as possible\n",
    "\n",
    "### This is not necessary for the experiments in our paper, it just speeds things up a bit."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "jobs = {'resnet18_00': 0,\n",
    "     \n",
    "#  'resnet18_100_argmax': 406060,\n",
    " 'resnet18_100_both': 208828,\n",
    " 'resnet18_100_cj_only': 140693,\n",
    " 'resnet18_100_cl_pbc': 208828,\n",
    " 'resnet18_100_cl_pbnr': 194125,\n",
    " 'resnet18_100random': 178294,\n",
    "     \n",
    " 'resnet18_20_argmax': 81212,\n",
    " 'resnet18_20_both': 41765,\n",
    " 'resnet18_20_cj_only': 28138,\n",
    " 'resnet18_20_cl_pbc': 41765,\n",
    " 'resnet18_20_cl_pbnr': 38825,\n",
    " 'resnet18_20random': 35658,\n",
    "     \n",
    " 'resnet18_40_argmax': 162424,\n",
    " 'resnet18_40_both': 83531,\n",
    " 'resnet18_40_cj_only': 56277,\n",
    " 'resnet18_40_cl_pbc': 83531,\n",
    " 'resnet18_40_cl_pbnr': 77650,\n",
    " 'resnet18_40random': 71317,\n",
    "     \n",
    "#  'resnet18_60_argmax': 243636,\n",
    " 'resnet18_60_both': 125296,\n",
    " 'resnet18_60_cj_only': 84415,\n",
    " 'resnet18_60_cl_pbc': 125296,\n",
    " 'resnet18_60_cl_pbnr': 116475,\n",
    " 'resnet18_60random': 106976,\n",
    "     \n",
    "#  'resnet18_80_argmax': 324848,\n",
    " 'resnet18_80_both': 167062,\n",
    " 'resnet18_80_cj_only': 112554,\n",
    " 'resnet18_80_cl_pbc': 167062,\n",
    " 'resnet18_80_cl_pbnr': 155300,\n",
    " 'resnet18_80random': 142635,\n",
    "}\n",
    "j = {v:k for k,v in jobs.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9465127\n",
      "2876671\n",
      "2419305\n",
      "2332495\n",
      "2141041\n",
      "2138857\n",
      "516735\n",
      "373593\n",
      "300429\n",
      "251386\n",
      "85664\n",
      "83679\n",
      "66663\n",
      "55373\n",
      "42236\n",
      "37144\n",
      "30856\n",
      "30211\n",
      "15849\n",
      "13551\n",
      "7201\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-130-b1b77bc1b836>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mwhile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mrands\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1281167\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjobs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrands\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mrands\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m0.25\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m0.25\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.75\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1.\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     \u001b[0mscore\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mscore\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mbest_score\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-130-b1b77bc1b836>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mwhile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mrands\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1281167\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjobs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrands\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mrands\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m0.25\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m0.25\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.75\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1.\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     \u001b[0mscore\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mscore\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mbest_score\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Stochastically searching for a balanced way distribution of workload\n",
    "# Kill once you think the (max - min) is low enough. Something around 7000 is good.\n",
    "best_rands = None\n",
    "best_score = np.inf\n",
    "while(True):\n",
    "    rands = np.random.rand(len(jobs))\n",
    "    scores = [sum((1281167 - np.array(list(jobs.values())))[(rands < i) & (rands >= (i - 0.25))]) for i in [0.25, 0.5, 0.75, 1.]]\n",
    "    score = np.max(scores) - np.min(scores)\n",
    "    if score < best_score:\n",
    "        best_score = score\n",
    "        best_rands = rands\n",
    "        print(score)\n",
    "        sys.stdout.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[755504, 761068, 756655, 762705]"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "partitions = [list(np.array(list(jobs.keys()))[(best_rands < i) & (best_rands >= (i - 0.25))]) for i in [0.25, 0.5, 0.75, 1.]]\n",
    "\n",
    "# Verify partitions are reasonable\n",
    "[np.sum([jobs[k] for k in p]) for p in partitions]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "GPU: 0\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_100_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_100_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_100random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_100random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_100random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_20_argmax\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_20_argmax\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_argmax_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_20_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_20_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_20_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_20_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_60_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_60_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_80_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_80_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 1\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_00\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_00\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_00.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_100_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_100_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_100_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_100_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_20random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_20random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_20random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_40_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_40_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_60_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_60_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_80_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_80_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 2\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_20_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_20_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_40_argmax\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_40_argmax\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_argmax_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_40_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_40_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_40_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_40_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_60_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_60_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_80_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_80_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_80_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_80_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 3\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_100_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_100_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_20_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_20_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_40_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_40_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_40random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_40random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_40random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_60_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_60_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_60random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_60random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_60random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_80random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial2/resnet18_80random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_80random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 0\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_100_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_100_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_100random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_100random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_100random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_20_argmax\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_20_argmax\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_argmax_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_20_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_20_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_20_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_20_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_60_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_60_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_80_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_80_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 1\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_00\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_00\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_00.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_100_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_100_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_100_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_100_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_20random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_20random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_20random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_40_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_40_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_60_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_60_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_80_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_80_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 2\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_20_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_20_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_40_argmax\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_40_argmax\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_argmax_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_40_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_40_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_40_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_40_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_60_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_60_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_80_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_80_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_80_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_80_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 3\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_100_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_100_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_20_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_20_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_40_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_40_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_40random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_40random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_40random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_60_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_60_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_60random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_60random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_60random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_80random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial3/resnet18_80random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_80random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 0\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_100_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_100_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_100random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_100random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_100random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_20_argmax\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_20_argmax\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_argmax_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_20_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_20_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_20_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_20_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_60_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_60_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_80_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_80_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 1\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_00\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_00\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_00.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_100_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_100_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_100_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_100_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_20random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_20random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_20random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_40_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_40_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_60_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_60_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_80_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_80_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 2\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_20_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_20_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_40_argmax\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_40_argmax\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_argmax_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_40_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_40_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_40_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_40_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_60_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_60_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_80_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_80_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_80_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_80_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 3\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_100_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_100_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_20_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_20_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_40_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_40_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_40random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_40random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_40random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_60_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_60_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_60random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_60random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_60random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_80random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial4/resnet18_80random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_80random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 0\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_100_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_100_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_100random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_100random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_100random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_20_argmax\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_20_argmax\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_argmax_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_20_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_20_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_20_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_20_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_60_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_60_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_80_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_80_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 1\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_00\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_00\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_00.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_100_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_100_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_100_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_100_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_20random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_20random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_20random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_40_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_40_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_60_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_60_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_80_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_80_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 2\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_20_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_20_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_40_argmax\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_40_argmax\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_argmax_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_40_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_40_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_40_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_40_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_60_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_60_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_80_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_80_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_80_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_80_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 3\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_100_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_100_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_20_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_20_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_40_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_40_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_40random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_40random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_40random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_60_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_60_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_60random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_60random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_60random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_80random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/trial5/resnet18_80random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 156 --gpu 3 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_80random.npy /datasets/datasets/imagenet/ >> out.log\n"
     ]
    }
   ],
   "source": [
    "# Generate jobs from partitions\n",
    "b = 156  # batch size\n",
    "for trial in [2,3,4,5]:\n",
    "    for g, p in enumerate(partitions):\n",
    "        print('\\nGPU: {}\\n'.format(g))\n",
    "        for f in p:\n",
    "            print('mkdir /home/cgn/masked_imagenet_training/resnet18/trial{}/{}'.format(trial, f))\n",
    "            print('cd /home/cgn/masked_imagenet_training/resnet18/trial{}/{}'.format(trial, f))\n",
    "            amt = f[9:].split('_')[0]\n",
    "            method = f[10 + len(amt):]\n",
    "            py = 'python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py'\n",
    "            params = ' -a \"resnet18\" --lr 0.1 -b {} --gpu {} '.format(b,g)\n",
    "            mask = '-m /home/cgn/masks/imagenet_train_bool_{}_mask__fraction_{}.npy '.format(method, amt)\n",
    "            suffix = '/datasets/datasets/imagenet/ >> out.log'\n",
    "            cmd = py + params + mask + suffix\n",
    "            print(cmd)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Now we do the same thing but we take advantage of the fact that with batch size 156 we can run two models on one GPU at a time. Its slower per model, but faster overall."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5870163\n",
      "3536332\n",
      "2437717\n",
      "1408746\n",
      "1395232\n",
      "1287516\n",
      "1215280\n",
      "1194018\n",
      "1170028\n",
      "1108224\n",
      "1107602\n",
      "1100124\n",
      "1056076\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-127-847ad7e0f02a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mwhile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mrands\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m     \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1281167\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjobs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrands\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mrands\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m0.125\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m0.125\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.25\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.375\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.625\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.75\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.875\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1.\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m     \u001b[0mscore\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mscore\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mbest_score\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-127-847ad7e0f02a>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mwhile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mrands\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m     \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1281167\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjobs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrands\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mrands\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m0.125\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m0.125\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.25\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.375\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.625\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.75\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.875\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1.\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m     \u001b[0mscore\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mscore\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mbest_score\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Stochastically searching for a balanced way distribution of workload\n",
    "# Kill once you the max and min scores are nearest too each other.\n",
    "# This is a much harder task because the number of bins is significantly higher.\n",
    "# A decent value might be 70,000.\n",
    "best_rands = None\n",
    "best_score = np.inf\n",
    "while(True):\n",
    "    rands = np.random.rand(len(jobs))\n",
    "    scores = [sum((1281167 - np.array(list(jobs.values())))[(rands < i) & (rands >= (i - 0.125))]) for i in [0.125, 0.25, 0.375, 0.5, 0.625, 0.75, 0.875, 1.]]\n",
    "    score = max(scores) - min(scores)\n",
    "    if score < best_score:\n",
    "        best_score = score\n",
    "        best_rands = rands\n",
    "        print(score)\n",
    "        sys.stdout.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[522535, 502556, 516205, 524874, 523252, 480148, 452463, 488443]"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "partitions = [list(np.array(list(jobs.keys()))[(best_rands < i) & (best_rands >= (i - 0.125))]) for i in [0.125, 0.25, 0.375, 0.5, 0.625, 0.75, 0.875, 1.]]\n",
    "\n",
    "# Verify partitions are reasonable\n",
    "[np.sum([jobs[k] for k in p]) for p in partitions]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['resnet18_100_argmax', 'resnet18_60_cl_pbnr'],\n",
       " ['resnet18_20_both',\n",
       "  'resnet18_20_cj_only',\n",
       "  'resnet18_40_argmax',\n",
       "  'resnet18_40_cj_only',\n",
       "  'resnet18_40random',\n",
       "  'resnet18_80random'],\n",
       " ['resnet18_100_cl_pbc',\n",
       "  'resnet18_20_argmax',\n",
       "  'resnet18_20random',\n",
       "  'resnet18_40_both',\n",
       "  'resnet18_60random'],\n",
       " ['resnet18_100_cj_only',\n",
       "  'resnet18_100random',\n",
       "  'resnet18_20_cl_pbnr',\n",
       "  'resnet18_80_both'],\n",
       " ['resnet18_00',\n",
       "  'resnet18_100_cl_pbnr',\n",
       "  'resnet18_40_cl_pbnr',\n",
       "  'resnet18_60_cj_only',\n",
       "  'resnet18_80_cl_pbc'],\n",
       " ['resnet18_80_argmax', 'resnet18_80_cl_pbnr'],\n",
       " ['resnet18_40_cl_pbc', 'resnet18_60_argmax', 'resnet18_60_both'],\n",
       " ['resnet18_100_both',\n",
       "  'resnet18_20_cl_pbc',\n",
       "  'resnet18_60_cl_pbc',\n",
       "  'resnet18_80_cj_only']]"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "partitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "GPU: 0\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_100_argmax\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_100_argmax\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_argmax_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_60_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_60_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 0\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_20_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_20_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_20_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_20_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_40_argmax\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_40_argmax\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_argmax_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_40_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_40_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 0 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_40random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_40random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 0 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_40random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_80random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_80random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 0 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_80random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 1\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_100_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_100_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_20_argmax\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_20_argmax\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_argmax_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_20random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_20random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 1 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_20random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_40_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_40_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_60random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_60random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 1 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_60random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 1\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_100_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_100_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_100random\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_100random\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 1 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_100random.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_20_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_20_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_80_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_80_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 1 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 2\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_00\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_00\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 2 -m /home/cgn/masks/imagenet_train_bool__mask__fraction_00.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_100_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_100_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_40_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_40_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_60_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_60_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_80_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_80_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 2\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_80_argmax\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_80_argmax\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_argmax_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_80_cl_pbnr\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_80_cl_pbnr\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 2 -m /home/cgn/masks/imagenet_train_bool_cl_pbnr_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 3\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_40_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_40_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_40.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_60_argmax\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_60_argmax\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_argmax_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_60_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_60_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "\n",
      "GPU: 3\n",
      "\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_100_both\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_100_both\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_both_mask__fraction_100.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_20_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_20_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_20.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_60_cl_pbc\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_60_cl_pbc\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_cl_pbc_mask__fraction_60.npy /datasets/datasets/imagenet/ >> out.log\n",
      "mkdir /home/cgn/masked_imagenet_training/resnet18/resnet18_80_cj_only\n",
      "cd /home/cgn/masked_imagenet_training/resnet18/resnet18_80_cj_only\n",
      "python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py -a \"resnet18\" --lr 0.1 -b 256 --gpu 3 -m /home/cgn/masks/imagenet_train_bool_cj_only_mask__fraction_80.npy /datasets/datasets/imagenet/ >> out.log\n"
     ]
    }
   ],
   "source": [
    "# Generate jobs from partitions\n",
    "for g, p in enumerate(partitions):\n",
    "    print('\\nGPU: {}\\n'.format(g // 2))\n",
    "    for f in p:\n",
    "        print('mkdir /home/cgn/masked_imagenet_training/resnet18/{}'.format(f))\n",
    "        print('cd /home/cgn/masked_imagenet_training/resnet18/{}'.format(f))\n",
    "        amt = f[9:].split('_')[0]\n",
    "        method = f[10 + len(amt):]\n",
    "        py = 'python3 /home/cgn/cgn/cleanlab/examples/imagenet/imagenet_train_crossval.py'\n",
    "        params = ' -a \"resnet18\" --lr 0.1 -b 256 --gpu {} '.format(g // 2)\n",
    "        mask = '-m /home/cgn/masks/imagenet_train_bool_{}_mask__fraction_{}.npy '.format(method, amt)\n",
    "        suffix = '/datasets/datasets/imagenet/ >> out.log'\n",
    "        cmd = py + params + mask + suffix\n",
    "        print(cmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
